package com.xiaotu.spider.project.pageprocessor.list;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import com.google.gson.Gson;
import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.KeywordEncode;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

/**
 * @类名 DouBanComListPageProcessor
 * @日期 2017年8月25日
 * @作者 王艳龙
 * @功能 豆瓣爬取类--TV 影评和短评
 */
@Component("DouBanRCListPageProcessor")
@Scope("prototype")
public class DouBanRCListPageProcessor extends AUrlPageProcessor
{
	
    private String fileName; //读取tvName文件路径
    private String fileReviews; //豆瓣影评
    private String fileComment; //豆瓣短评
    private String movieUrl; //不匹配电影URL
    
    @Override
    protected String[] getStartRequestUrls()
    {
        List<JSONObject> targetList = readFileColumn(fileName); //读取excel电影名
        List<String> linkList = new ArrayList<String>();
        for (JSONObject target : targetList)
        {
            String tvName = target.getString("tvName");
            String startDate = target.getString("startDate");
            linkList.add("https://www.douban.com/search?cat=1002&q=" + KeywordEncode.encode(tvName) + "&startDate=" +
                    startDate);
        }
        String[] urlArr = new String[linkList.size()];
        linkList.toArray(urlArr);
        return urlArr;

    }

    @Override
    public void initRunParams(Map<String, Object> map)
    {
        if (map.get("fileName") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        fileName = map.get("fileName") + StringUtils.EMPTY;

        if (map.get("fileReviews") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        fileReviews = map.get("fileReviews") + StringUtils.EMPTY;

        if (map.get("fileComment") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        fileComment = map.get("fileComment") + StringUtils.EMPTY;
        
        if (map.get("movieUrl") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        movieUrl = map.get("movieUrl") + StringUtils.EMPTY;
    }

    @Override
    public String getPageDataType(Page page)
    {
        String url = page.getUrl().toString();
        if (url.contains("https://www.douban.com/search?cat=1002&q=")){
        	return Constants.PageDataType.SEARCH_LIST;
        	
        }else if (url.contains("https://www.douban.com/link2/")){
        	return Constants.PageDataType.TV_PRO_COMMENT;
        	
        }else if(url.contains("/comments")){
        	 return Constants.PageDataType.TV_COMMENT;
        	 
        }else if (url.contains("/reviews"))
            return Constants.PageDataType.TV_REVIEWS;
        return Constants.PageDataType.TV_PRO_COMMENT;
    }

    @Override
    public void process(Page page)
    {
        String url = page.getUrl().toString();
        Map<String, Object> tvMap = page.getRequest().getExtras();
        if (url.length() > 0 && url.contains("&q="))
        {
            String tvName = url.substring(url.indexOf("&q=") + 3, url.indexOf("&startDate"));
            String startDate ="";
            if(url.contains(".0")){
            	 startDate = url.substring(url.indexOf("Date=") + 5, url.indexOf(".0"));
            }else{
            	 startDate = url.substring(url.indexOf("Date=") + 5, url.length());
            	
            }
            tvMap.put("tvName", KeywordEncode.decode(tvName));
            tvMap.put("startDate", startDate);
        }
        switch (this.getPageDataType(page))
        {
            case Constants.PageDataType.SEARCH_LIST:
                this.searchListProcess(page, tvMap);
                break;
            case Constants.PageDataType.TV_PRO_COMMENT:
                this.reviewCommentProcess(page, tvMap);
                break;
            case Constants.PageDataType.TV_COMMENT:
                this.commentProcess(page, tvMap);
                break;
            case Constants.PageDataType.TV_REVIEWS:
                this.reviewsProcess(page, tvMap);
                break;
            default:
                break;
        }
    }

    private void searchListProcess(Page page, Map<String, Object> tvMap)
    {
    	boolean logType =true;
    	String status ="1";
        //获取搜索也页面列表
        final List<String> searchListHTML = page.getHtml()
                .xpath("//div[@class='result']/div[@class='content']").all();
        for (String htmlStr : searchListHTML)
        {
            Html html = new Html(htmlStr);
            //获取影片
            String movieType = html.xpath("//div[@class='title']/h3/span/text()").toString();
            //获取影片名称
            String movieName = html.xpath("//div[@class='title']/h3/a/text()").toString().trim();
            //获取影片年代
            String movieAges = html.xpath("//div[@class='rating-info']/span[@class='subject-cast']/text()").toString();
            if (movieAges != null && movieAges != "")
            {
            	movieAges = movieAges.substring(movieAges.length() - 4, movieAges.length());
                tvMap.put("movieAge", movieAges);
            }
            String tvMapName = (String) tvMap.get("tvName");
            String startDate =(String) tvMap.get("startDate");
            tvMap.put("movieName", movieName);
            String links = html.xpath("//div[@class='title']/h3/a/@href").get();
            if (movieType.equals("[电影]") && movieName.equals(tvMapName) && movieAges.equals(startDate)){
            	status ="0";
                tvMap.put("linkUrl", links);
                //存放影片地址
                PageUtils.addTargetRequest(page, links, page.getRequest().getExtras());
            }
        }
	        if(logType && status=="1"){
	        	FileUtil.writeFile(movieUrl, page.getUrl().toString()
	    				+ SepratorUtil.LINE_SEPRATOR);
	    		logType =false;
	    	}
    }


    private void reviewCommentProcess(Page page, Map<String, Object> tvMap){
    	//获取影评地址
		String reviewUrls = page.getHtml().xpath("//section[@class='reviews mod " +
				"movie-content']/header/a/@data-verify-url").get();
		if (StringUtils.isNotEmpty(reviewUrls))
		{
			String reviewUrl = reviewUrls.substring(reviewUrls.indexOf("redir=") + 6, reviewUrls.indexOf("new_review")) + "reviews";
			PageUtils.addTargetRequest(page, reviewUrl, page.getRequest().getExtras());
		}
		//获取短评地址
		String comUrls = page.getHtml().xpath("//div[@id='comments-section']/div[@class='mod-hd']/h2/span/a/@href").get();
		if (StringUtils.isNotEmpty(comUrls))
		{
			PageUtils.addTargetRequest(page, comUrls, page.getRequest().getExtras());
		}
}

    private void commentProcess(Page page, Map<String, Object> tvMap){
    	List<String> divList = page.getHtml()
				.xpath("//div[@class='comment-item']").all();
		if (divList == null || divList.isEmpty())
			throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		for (String divStr : divList) {
			Html divObj = new Html(divStr);
			//电影名称
			String movieName = (String) tvMap.get("movieName");
			if (movieName == null || movieName.isEmpty())
				return;
			//评论人
			String comReviewer = divObj.xpath("//div[@class='avatar']/a/@title")
					.get();
			if (comReviewer == null || comReviewer.isEmpty())
				return;
			//评论时间
			String comTime = divObj.xpath("//span[@class='comment-time']/@title")
					.get();
			if (comTime == null || comTime.isEmpty())
				return;
			//评论内容
			String comContent = divObj.xpath("//div[@class='comment']/p/text()")
					.get();
			if (comContent == null || comContent.isEmpty())
				return;
			//评论权重
			StringBuffer buffer = new StringBuffer();
			String votes = divObj.xpath("//span[@class='comment-vote']/span[@class='votes']/text()").get();
			if (votes == null || votes.isEmpty())
				return;
			String show_login = divObj.xpath(
					"//span[@class='comment-vote']/a/text()").get();
			if (show_login == null || show_login.isEmpty())
				return;
			String comment_votes = buffer.append(votes)
					.append(SepratorUtil.SEP_SPACE_EN).append(show_login).toString();
			//评论等级	
			String comLevel="";
			List<String> comLevelClass = divObj.xpath(
					"//span[@class='comment-info']/span/@class").all();
			for(String cla: comLevelClass){
				if(cla.contains("allstar")){
					comLevel = cla.substring(cla.indexOf("allstar")+7, cla.indexOf("0 rating"));
				}
			}
			
			String line = StringUtils.join(new String[]{StringUtil.nullToString(page.getRequest().getExtra("type")),
					movieName, comReviewer,comTime,comContent, 
					comment_votes,comLevel}, SepratorUtil.SEP_PIPE);
	        System.out.println(line);
	        FileUtil.writeFile(fileComment, line + SepratorUtil.LINE_SEPRATOR);
	        
		}
		//无条数限制，爬取所有数据
		String current = page.getHtml()
				.xpath("//div[@class='center']/a[@class='next']/@href").get();
		if (current == null || current.isEmpty())
			return;
		if (StringUtils.isNotEmpty(current)){
			PageUtils.addTargetRequest(page,current,page.getRequest().getExtras());
		}
       
    }
    
    private void reviewsProcess(Page page, Map<String, Object> tvMap)
    {
        StringBuffer buffer = new StringBuffer();
        // 获取专业影评
        List<Selectable> proCommList = page.getHtml()
                .xpath("//div[@class='review-list']/div").nodes();

        if (null == proCommList || proCommList.size() < 1)
            throw new SpiderException(
                    SpiderExceptionCode.DataInvalideException);

        for (Selectable procmt : proCommList)
        {

            //电影名称
            String movieName = (String) tvMap.get("tvName");
            buffer.append(movieName);
            //电影年份
    		String movieAge =(String)tvMap.get("movieAge");
    		buffer.append(SepratorUtil.SEP_PIPE+movieAge);
            //影评发布者
            String movieReviewers = procmt.xpath("//div[@class='header-more']/a/span/text()").toString();
            if(StringUtils.isEmpty(movieReviewers)){
            	movieReviewers =SepratorUtil.SEP_SPACE_EN;
            }
            if(movieReviewers.contains("|")){
            	movieReviewers =movieReviewers.replace("|", "");
            }
            buffer.append(SepratorUtil.SEP_PIPE + movieReviewers);

            //影评发布时间
            String movieReleaseTime = procmt.xpath("//div[@class='header-more']/span[@class='main-meta']/text()")
                    .toString();
            if(StringUtils.isEmpty(movieReleaseTime)){
            	movieReleaseTime =SepratorUtil.SEP_SPACE_EN;
            }
            buffer.append(SepratorUtil.SEP_PIPE + movieReleaseTime);

            //影评标题
            String movieTitle = procmt.xpath("//h3[@class='title']/a/text()").toString();
            if(StringUtils.isEmpty(movieTitle)){
            	movieTitle =SepratorUtil.SEP_SPACE_EN;
            }
            buffer.append(SepratorUtil.SEP_PIPE + movieTitle);

            //影评星级
            String movieStar = procmt.xpath("//div[@class='header-more']/span[@property='v:rating']/@class")
                    .toString();
            if (StringUtils.isEmpty(movieStar) || StringUtils.isEmpty(movieStar.trim())){
            	movieStar =SepratorUtil.SEP_SPACE_EN;
            	buffer.append(SepratorUtil.SEP_PIPE+movieStar);
            }else{
            	movieStar = movieStar.substring(movieStar.indexOf("allstar") + 7, movieStar.indexOf("0 " +
            			"main-title-rating"));
            	buffer.append(SepratorUtil.SEP_PIPE + movieStar);
            }

            //影评内容
            String comment = procmt
                    .xpath("//div[@class='short-content']/text()").toString();
            if (StringUtils.isEmpty(comment) || StringUtils.isEmpty(comment.trim())){
            	comment ="没关系,可以显示全文";
            }
            	buffer.append(SepratorUtil.SEP_PIPE + comment);
            
            //回复评论人数
            String movieUseNum = procmt.xpath("//div[@class='main-bd']/div/div[@class='short-content']/a/text()")
                    .toString();
            if (StringUtils.isEmpty(movieUseNum) || StringUtils.isEmpty(movieUseNum.trim())){
            	movieUseNum =SepratorUtil.SEP_SPACE_EN;
            }
            buffer.append(SepratorUtil.SEP_PIPE + movieUseNum);

            //有用 没有人数
            String up = null, down = null;
            String upDown = procmt.xpath("//span[@class='left']/text()")
                    .toString();
            if (StringUtils.isNotEmpty(upDown))
            {
                up = RegexUtils.regexSearch("([0-9]+)有用", upDown);
                up = StringUtils.isEmpty(up) ? up
                        : RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, up);
                if (StringUtils.isEmpty(up)){
                	up =SepratorUtil.SEP_SPACE_EN;
                }
                buffer.append(SepratorUtil.SEP_PIPE + up);

                down = RegexUtils.regexSearch("([0-9]+)没用", upDown);
                down = StringUtils.isEmpty(down) ? down
                        : RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER,
                        down);
                if (StringUtils.isEmpty(down)){
                	down =SepratorUtil.SEP_SPACE_EN;
                }
                buffer.append(SepratorUtil.SEP_PIPE + down);
            }

            FileUtil.writeFile(fileReviews, buffer.toString() + SepratorUtil.LINE_SEPRATOR);
            buffer.setLength(0);
        }
        
        String current = page.getHtml()
                .xpath("//div[@class='paginator']/span[@class='next']/a/@href").get();
		if (current == null || current.isEmpty())
			return;
		if (StringUtils.isNotEmpty(current)){
			PageUtils.addTargetRequest(page,current,page.getRequest().getExtras());
		}
       /* int pageMax = 200;
        String current = page.getHtml()
                .xpath("//div[@class='paginator']/span[@class='next']/a/@href").get();
        if (current != null)
        {
            String pageSize = current.substring(current.indexOf("start=") + 6, current.length());
            if (StringUtils.isNotEmpty(pageSize))
            {
                pageSize = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, pageSize);
                if (StringUtils.isNotEmpty(pageSize))
                {
                    if (pageMax >= Integer.parseInt(pageSize))
                        PageUtils.addTargetRequest(page,
                                current,
                                page.getRequest().getExtras());
                }
            }
        }*/
    }
    /**
     * 读取文件指定列
     *
     * @param fileName 文件名称
     * @return 行列表
     */
    public static List<JSONObject> readFileColumn(String fileName)
    {
        Map<String, Object> map = new HashMap<String, Object>();
        Workbook wb = null;
        String json;
        List<JSONObject> list = new ArrayList<JSONObject>();
        try
        {
            InputStream inputStream = new FileInputStream(fileName);
            if (fileName.endsWith("xls"))
            {
                wb = new HSSFWorkbook(inputStream); // 解析xls格式
            } else if (fileName.endsWith("xlsx"))
            {
                wb = new XSSFWorkbook(inputStream); // 解析xlsx格式
            }
            Sheet sheet = wb.getSheetAt(0); // 第一个工作表

            int firstRowNum = 1;
            int lastRowNum = sheet.getLastRowNum();

            Row row = null;
            Cell cell_a1 = null;
            Cell cell_a5 = null;
            for (int i = firstRowNum; i <= lastRowNum; i++)
            {
                row = sheet.getRow(i);          //取得第i行
                cell_a1 = row.getCell(1);      //取得i行的第一列
                cell_a5 = row.getCell(2);
                if(row.getCell(1)!=null){
                	cell_a1.setCellType(Cell.CELL_TYPE_STRING);
                    String cellValue1 = cell_a1.getStringCellValue();
                    map.put("tvName", cellValue1);
               }
                if(row.getCell(2)!=null){
                	cell_a5.setCellType(Cell.CELL_TYPE_STRING);
                    String cellValue5 = cell_a5.getStringCellValue().trim();
                    map.put("startDate", cellValue5);
               }
                json = new Gson().toJson(map);
                JSONObject jsStr = JSONObject.fromObject(json);
                list.add(jsStr);
            }

        }
        catch (FileNotFoundException e)
        {
            e.printStackTrace();
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        return list;
    }

    @Override
    protected String getSpiderType()
    {
        return "DouBanRCList";
    }
}
